from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from multiprocessing import Process
from optparse import OptionParser
from multiprocessing import Pool
from pymongo import MongoClient
from selenium import webdriver
from bs4 import BeautifulSoup
from urllib import request
from urllib import parse

import requests
import hashlib
import pprint
import base64
import random
import time
import json

parser = OptionParser(usage = "%prog [options]")

parser.add_option("-e", "--net", action = "store", type = "str", dest = "net", help = "the ip", default = '127.0.0.1')
parser.add_option("-n", "--count", action = "store", type = "int", dest = "count", help = "the count", default = 10)

options, args = parser.parse_args()

print('=' * 40)
print(options.net)
print('=' * 40)

client = MongoClient(options.net, 27017, connect = False)

lagou_comp_db = client['lagou_comp_db']
lagou_comp_coll = lagou_comp_db['lagou_comp_coll']
lagou_comp_coll_no_exists = lagou_comp_db['lagou_comp_coll_no_exists']

# 代理IP地址
# ip_api = 'http://api.qingtingip.com/ip?app_key=f90d5ea37ebfbd5a47482cf777196805&num=1&ptc=http&fmt=text&lb=\r\n&port=0&mr=2&'
ip_api = ''

pcUserAgent = [
    # safari 5.1 – Windows
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    # Firefox 38esr
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
    # IE 11
    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
    # IE 9.0
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
    # IE 8.0
    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
    # IE 7.0
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    # IE 6.0
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
    # Firefox 4.0.1 – MAC
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    # Firefox 4.0.1 – Windows
    "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    # Opera 11.11 – MAC
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
    # safari 5.1 – MAC
    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    # Green Browser
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    # Avant
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
    # 360浏览器
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    # 搜狗浏览器 1.x
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    # 世界之窗（The World） 3.x
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
    # 世界之窗（The World） 2.x
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    # 腾讯TT
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
    # 傲游（Maxthon）
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
    # Chrome 17.0 – MAC
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    # Opera 11.11 – Windows
    "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11"
]


def new_driver(headless = False):
    if ip_api:
        time.sleep(random.randint(0, 5))

        try:
            r = requests.get(ip_api)
            proxy = r.text.strip()
        except:
            return new_driver(headless)
    else:
        proxy = ''

    chrome_options = webdriver.ChromeOptions()

    if proxy:
        chrome_options.add_argument('--proxy-server=http://' + proxy)
        print(proxy)

    if headless:
        chrome_options.add_argument('--headless')

    chrome_options.add_argument('--start-maximized')
    chrome_options.add_argument('user-agent=' + random.choice(pcUserAgent) + str(random.randint(0, 10000)))
    chrome_options.add_argument(
        'Proxy-Authorization=Basic ' + str(base64.b64encode('651798768@qq.com:123456aaa'.encode('utf-8')), 'utf-8'))
    chrome_driver_path = '/Users/xuchaosheng/Workspace/cv_scrapy/libs/chromedriver'
    driver = webdriver.Chrome(chrome_options = chrome_options, executable_path = chrome_driver_path)
    driver.get('https://passport.lagou.com/login/login.html')
    return driver


driver = None


def lagou_company_detail(id, driver_index):
    global driver

    url = 'https://www.lagou.com/gongsi/' + str(id) + '.html'

    try:
        driver.get(url)
    except:
        # driver.quit()
        # driver = new_driver(headless = False)
        time.sleep(5)
        return lagou_company_detail(id, driver_index)

    time.sleep(2)

    if 'https://passport.lagou.com/login/login.html' in driver.current_url:
        # driver.quit()
        # driver = new_driver(headless = False)
        time.sleep(30)
        return lagou_company_detail(id, driver_index)

    page_source = driver.page_source

    if '无法访问此网站' in page_source or '未连接到互联网' in page_source or '您已被封禁' in page_source or '当前账号存在异常，请验证后继续访问' in page_source:
        # driver.quit()
        # driver = new_driver(headless = False)

        print('=' * 100)
        print('快来打码啊')
        print('=' * 100)

        time.sleep(30)
        return lagou_company_detail(id, driver_index)

    if '本网站的Coder和PM私奔啦' in page_source or '这个公司的主页还在建设中' in page_source:
        lagou_comp_coll_no_exists.insert_one({'id': id})
        return

    page_source = driver.page_source

    soup = BeautifulSoup(page_source, 'lxml')

    if not soup.find(class_ = 'company_intro_text'):
        print('=' * 100)
        print(page_source)
        print('=' * 100)
        return

    data = {
        'description': soup.find(class_ = 'company_intro_text').get_text().strip(),
        'shortname': soup.find(class_ = 'company_info').find('h1').find('a').get_text().strip(),
        'fullname': soup.find(class_ = 'company_info').find('h1').find('a').get('title'),
        'industry': soup.find(id = 'basic_container').find(class_ = 'type').find_next_sibling().text,
        'financing': soup.find(id = 'basic_container').find(class_ = 'process').find_next_sibling().text,
        'employees': soup.find(id = 'basic_container').find(class_ = 'number').find_next_sibling().text,
        'city': soup.find(id = 'basic_container').find(class_ = 'address').find_next_sibling().text if soup.find(
            id = 'basic_container').find(class_ = 'address') else '',
        'products': [],
        'id': id,
        'events': [],
        'slgon': soup.find(class_ = 'company_word').get_text().strip(),
        'logo': 'http:' + soup.find(class_ = 'top_info').find('img').get('src')
    }

    for item in soup.select('#company_products .product_item'):
        data['products'].append({
            'name': item.find('h4').get_text().strip(),
            'description': item.find(class_ = 'product_profile').get_text().strip(),
            'type': [i.get_text() for i in item.select('ul li')]
        })

    for item in soup.select('.history_ul .history_li'):
        data['events'].append({
            'date': item.find(class_ = 'li_date').get_text().strip() if item.find(class_ = 'li_date') else '',
            'title': item.find(class_ = 'desc_title').get_text().strip() if item.find(class_ = 'desc_title') else '',
            'desc': item.find(class_ = 'desc_intro').get_text().strip() if item.find(class_ = 'desc_intro') else ''
        })

    return data


def start(start, end, headless, driver_index):
    global driver
    driver = new_driver()

    time.sleep(100)

    count = {}

    for id in range(end, start, -1):
        # rand_id = random.randint(start, end)
        rand_id = id

        if rand_id in count:
            continue

        if lagou_comp_coll_no_exists.find_one({'id': rand_id}):
            print('%d is not exists' % rand_id)
            continue

        if lagou_comp_coll.find_one({'id': rand_id}):
            print('%d is exists' % rand_id)
            continue

        item = lagou_company_detail(rand_id, driver_index)

        if item:
            lagou_comp_coll.insert_one(item)
            pprint.pprint(item)

        count[rand_id] = 1

        time.sleep(4)


if __name__ == '__main__':
    ps = []

    step = int(300000 / options.count)

    for p in range(options.count):
        headless = False
        ps.append(Process(target = start, args = (p * step, (p + 1) * step, headless, p)))

    for p in ps:
        p.start()

    for p in ps:
        p.join()
